from sklearn.model_selection import learning_curve
def get_metrics_binary(classifier, y_test, y_pred, X_train, y_train):
"""
Function that calculates evaluation metrics for a binary classification
algorithm, generates the metrics table, and displays ROC, Precision-Recall,
learning curve, and confusion matrix plots.
Inputs:
- classifier: Classification estimator.
- X_test: Feature matrix of the test set.
- y_test: Target variable vector in the test set.
- y_pred: Predicted target variable vector in the test set.
- X_train: Feature matrix of the training set.
- y_train: Target variable vector in the training set.
Outputs:
- metrics: Table with classifier metrics.
- cm_df: Confusion matrix DataFrame.
- train_scores: Training scores from the learning curve.
- test_scores: Test scores from the learning curve.
- roc_auc: Area under the ROC curve.
"""
# Compute and plot the learning curve
print('Learning curve: \n')
train_sizes, train_scores, test_scores = learning_curve(classifier,
X_train, y_train, cv=5)
train_scores_mean = np.mean(train_scores, axis=1)
plt.figure(figsize=(8, 6))
plt.plot(train_sizes, train_scores_mean, 'o-', label='Training Accuracy')
plt.xlabel('Training Examples')
plt.ylabel('Accuracy')
plt.title('Learning Curve')
plt.legend(loc="lower right")
plt.show()
# Save learning curve values
class_name = [f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}']
df_learning_curve = {
'model': class_name,
'train_sizes': train_sizes,
'train_scores': train_scores_mean
}
df_learning_curve = pd.DataFrame(df_learning_curve)
print('\n------------------------------------------------------------\n')
# Compute and display the confusion matrix
print('Confussion matrix: \n')
cm = confusion_matrix(y_test, y_pred)
cm_df = pd.DataFrame(cm, index=['Actual Negative', 'Actual Positive'],
columns=['Predicted Negative', 'Predicted Positive'])
plt.figure(figsize=(8, 6))
sns.heatmap(cm_df, annot=True, fmt='d', cmap='Blues')
plt.title('Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()
print('\n------------------------------------------------------------\n')
# Calculate metrics
print('Relevant metrics: \n')
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)
# Create metrics table
df_metrics = {
'Accuracy': accuracy,
'Precision': precision,
'Recall': recall,
'F1-Score': f1,
'ROC-Area': roc_auc_score(y_test, y_pred)
}
df_metrics = pd.DataFrame(df_metrics, index=[f'{classifier.__class__.__name__}'])
display(df_metrics)
print('\n------------------------------------------------------------\n')
# Compute and plot the ROC curve
print('ROC and precision-recall curve:\n')
fpr, tpr, thresholds = roc_curve(y_test, y_pred)
roc_auc = roc_auc_score(y_test, y_pred)
# Create subplots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6))
# Plot ROC curve
ax1.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
ax1.plot([0, 1], [0, 1], 'k--')
ax1.set_xlim([0.0, 1.0])
ax1.set_ylim([0.0, 1.05])
ax1.set_xlabel('False Positive Rate')
ax1.set_ylabel('True Positive Rate')
ax1.set_title('Receiver Operating Characteristic')
ax1.legend(loc="lower right")
# Compute and plot the Precision-Recall curve
precision, recall, _ = precision_recall_curve(y_test, y_pred)
# Plot Precision-Recall curve
ax2.plot(recall, precision, label='Precision-Recall curve')
ax2.set_xlabel('Recall')
ax2.set_ylabel('Precision')
ax2.set_title('Precision-Recall Curve')
ax2.legend(loc="lower right")
# Adjust the spacing between subplots
plt.tight_layout()
# Show the plot
plt.show()
# Save presicion-recall curve values
df_prc = {
'precision': precision,
'recall': recall}
df_prc = pd.DataFrame(df_prc)
# Save auc curve values
class_name = [f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}',
f'{classifier.__class__.__name__}']
df_roc = {
'model': class_name,
'fpr': fpr,
'tpr': tpr,
'roc-auc': roc_auc
}
df_roc = pd.DataFrame(df_roc)
print('\n------------------------------------------------------------\n')
return df_metrics, cm_df, df_learning_curve, df_roc, df_prc
